import requests
import json

class BilibiliSpider:
    def __init__(self):
        # self.start_url = "https://api.bilibili.com/x/web-interface/newlist?rid=22&type=0&pn=1&ps=20"
        self.url_temp = "https://api.bilibili.com/x/web-interface/newlist?rid=22&type=0&pn={}&ps=20"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
            "Host": "api.bilibili.com",
            "Referer": "https://www.bilibili.com/v/kichiku/guide/?spm_id_from=333.5.b_7375626e6176.2"
        }
        self.number = 0

    def parse_url(self,url):
        print(url)
        r = requests.get(url,headers=self.headers)
        return r.content.decode()

    def get_content_list(self,html_str):
        json_str = json.loads(html_str)
        content_list = json_str["data"]["archives"]
        count = json_str["data"]["page"]["count"]
        num = json_str["data"]["page"]["num"]
        size = json_str["data"]["page"]["size"]
        return content_list,count,num,size

    def save_file(self,content_list):
        with open("哔哩哔哩json.txt","a",encoding="utf_8") as f:
            for content in content_list:
                f.write(json.dumps(content,ensure_ascii=False))
                f.write("\n")
        print("保存成功！")

    def run(self):
        # # 1.start_url
        # # 2.发送请求，接受响应
        # start_str = self.parse_url(self.start_url)
        # # 3.提取数据
        # content_list, count, num, size = self.get_content_list(start_str)
        # # 4.保存
        # self.save_file(content_list)
        # # 5.翻页
        # count = int(count)
        # num = int(num)
        # size = int(size)
        # while num*size < count:
        #     self.parse_url(self.url_temp.format(str(num+1)))
        #     content_list, count, num, size = self.get_content_list(start_str)
        #     self.save_file(content_list)

        count = 100 #假设有下一页
        num = 0
        size = 20
        while num*size < count:
            html_str = self.parse_url(self.url_temp.format(str(num + 1)))
            content_list, count, num, size = self.get_content_list(html_str)
            self.save_file(content_list)
            count = int(count)
            num = int(num)
            size = int(size)
            self.number += 1
            if self.number>1:
                break

if __name__ == '__main__':
    bilibili_spider = BilibiliSpider()
    bilibili_spider.run()